from selenium import webdriver
import time
from bs4 import BeautifulSoup
from urllib import parse
import lxml

browser = webdriver.Chrome()
# 设置等待时长
browser.implicitly_wait(10)
# print(driver.page_source) 打印整个页面的信息

# 进入国家统计局年度数据
browser.get("http://data.stats.gov.cn/easyquery.htm?cn=C01")

# 通过类名查找文件，注意find_element和find_elements的区别，一个只查找一个元素
# categories = browser.find_elements_by_id("treeZhiBiao")

# 获取各总类，通过a查找而不是li查找，a可以实现下拉操作
category_xpath = "//*[@id='treeZhiBiao__span']"
categories = browser.find_elements_by_xpath(category_xpath)

# 爬取单类内容
single_category_xpath = "//*[@id='treeZhiBiao_10_span']"
category = browser.find_element_by_xpath(single_category_xpath)
category.click()

# 价格指数部分,此处为点击价格指数，获取其子类内容
single_category_xpath = "//*[@id='treeZhiBiao_10_ul']/li"
lis = browser.find_elements_by_xpath(single_category_xpath)
# for li in lis:
#     li.click()
#     # 需等待一定时间，加载与消除蒙层内容
#     time.sleep(5)

# 单页数据爬取
pages = lis[1].click()
soup = BeautifulSoup(browser.page_source,"lxml")
# 获取横坐标内容
x = soup.find("//*[@id='main-container']/div[2]/div[2]/div[2]/div/div[1]/table/thead/tr/th")
x = browser.find_elements_by_xpath("//*[@id='main-container']/div[2]/div[2]/div[2]/div/div[1]/table/thead/tr/th")
# 获取纵坐标内容
trs = browser.find_elements_by_xpath("//*[@id='main-container']/div[2]/div[2]/div[2]/div/div[3]/table/tbody/tr")
for tr in trs:
    values = tr.find_elements_by_css_selector('td')
    index = ""
    value = []
    # print(len(values))
    for i in range(0,len(values)):
        if(i==0):
            index = values[i].text
        #此处需要注意，直接使用text无法获取，数字内容被隐藏，需要使用获取属性方法通过innerHTML获取值
        else:
            print(values[i].get_attribute("innerHTML"))
            print(len(values[i].get_attribute("innerHTML")))
            value.append(float(values[i].get_attribute("innerHTML")) if values[i].get_attribute("innerHTML")!="" else None)
    print(index)
    print(value)
# 自动化爬取所有总类
for category in categories:
    # category.click()
    time.sleep(1)
    print(category.tag_name)


# for category in categories:
#     category.click()
# browser.close()